

#从数据库中读取结构化的文本数据

#按照规则进行分块、组合

#返回最终的文本块

#输入：要拆分哪篇文章
#输出：这篇文章所有的文本块，其中文本块要符合结构化的要求，即按段拆分、且所有节点都要包含祖先节点的信息

#数据处理在导入mysql数据库时完成，此处只进行数据提取导入到向量数据库


import pymysql
from langchain.docstore.document import Document
from config import cfg

DATABASE_HOST = cfg.readValue('DATABASE', 'host')
DATABASE_PORT = cfg.readValue('DATABASE', 'port')
DATABASE_USER = cfg.readValue('DATABASE', 'user')
DATABASE_PASSWORD = cfg.readValue('DATABASE', 'password')
DATABASE_DB = cfg.readValue('DATABASE', 'db')



def structual_text(article_name):

    conn = pymysql.connect(host=DATABASE_HOST,user=DATABASE_USER,password=DATABASE_PASSWORD,db=DATABASE_DB,port=DATABASE_PORT)
    cursor = conn.cursor()
    sql = "select txt_id from documents_article_info where article_name=%s"
    try:
        cursor.execute(sql,(article_name))
        # 获取所有记录列表
        result = cursor.fetchall()
    except:
        conn.rollback()
        return "没有记录"

    sql = "SELECT text_with_title,article_id,owner from documents_read_info_mid where article_id = %s"
    #sql = "INSERT INTO documents_read_info (article_id,read_id,read_text,read_flag,owner) VALUES ( %s, %s,%s, %s,%s) "
    
    try:
        # 执行SQL语句
        #cursor.execute(sql, (article_id, read_id, read_text, flag, owner))
        cursor.execute(sql,(result))
        # 获取所有记录列表
        results = cursor.fetchall()

    except:
        conn.rollback()
        return "获取失败"
    
    finally:
        # 关闭数据库连接
        conn.close()
    
    documents = []
    for result in results:
        flag = 1
        # result = str(result[0].replace("\n",""))
        result = str(result[0])

        # print(result)
        while flag:
            flag = 1 if result.startswith("|") else 0
            if flag:
                result = result[1:]
            else:
                break
        re = result.split("|||")
        # print(re)
        cont = re[-1]
        # print(cont)
        head = re[:len(re)-1]
        # print(head)
        document = Document(page_content=" ".join(re), metadata={'source': article_name,'ans':''})
        documents.append(document)
        document = Document(page_content=" ".join(head), metadata={'source': article_name,'ans':cont})
        documents.append(document)
    
    return documents


def structual_text_v2(article_name):
    conn = pymysql.connect(host=DATABASE_HOST, user=DATABASE_USER, password=DATABASE_PASSWORD, db=DATABASE_DB,
                           port=DATABASE_PORT)
    cursor = conn.cursor()
    sql = "select txt_id from documents_article_info where article_name=%s"
    try:
        cursor.execute(sql, (article_name))
        # 获取所有记录列表
        result = cursor.fetchall()
    except:
        conn.rollback()
        return "没有记录"

    sql = "SELECT text_with_title,article_id,owner,page from documents_read_info_mid where article_id = %s"
    # sql = "INSERT INTO documents_read_info (article_id,read_id,read_text,read_flag,owner) VALUES ( %s, %s,%s, %s,%s) "

    try:
        # 执行SQL语句
        # cursor.execute(sql, (article_id, read_id, read_text, flag, owner))
        cursor.execute(sql, (result))
        # 获取所有记录列表
        results = cursor.fetchall()

    except:
        conn.rollback()
        return "获取失败"

    finally:
        # 关闭数据库连接
        conn.close()

    documents = []
    for item in results:
        flag = 1
        page = item[3]
        # print(f"page:{page}")
        result = str(item[0])
        while flag:
            flag = 1 if result.startswith("|") else 0
            if flag:
                result = result[1:]
            else:
                break
        re = result.split("|||")
        come = article_name+"_"+page
        # print(f"come:{come}")
        # print(f"re:{re}")
        document = Document(page_content=" ".join(re), metadata={'source': come, 'ans': ''})
        documents.append(document)
        cont = re[-1]
        # print(f"cont:{cont}")
        head = re[:len(re) - 1]
        # print(f"head:{head}")
        document = Document(page_content=" ".join(head), metadata={'source': come, 'ans': cont})
        documents.append(document)
        # break

    return documents


def structual_text_v4(article_name):
    conn = pymysql.connect(host=DATABASE_HOST, user=DATABASE_USER, password=DATABASE_PASSWORD, db=DATABASE_DB,
                           port=DATABASE_PORT)
    cursor = conn.cursor()
    sql = "select txt_id,old_article_name,article_name from documents_article_info where old_article_name=%s"
    try:
        cursor.execute(sql, (article_name))
        # 获取所有记录列表
        result = cursor.fetchone()
    except:
        conn.rollback()
        return "没有记录"


    sql = "SELECT text_with_title,article_id,owner from documents_read_info_mid where article_id = %s"
    # sql = "INSERT INTO documents_read_info (article_id,read_id,read_text,read_flag,owner) VALUES ( %s, %s,%s, %s,%s) "

    try:
        # 执行SQL语句
        # cursor.execute(sql, (article_id, read_id, read_text, flag, owner))
        cursor.execute(sql, (result[0]))
        # 获取所有记录列表
        results = cursor.fetchall()

    except:
        conn.rollback()
        return "获取失败"

    finally:
        # 关闭数据库连接
        conn.close()
    documents = []
    for res in results:
        content = res[0].strip()
        path = f"/app/create_docx/out_pdf/{article_name}.pdf"
        org_article_name = result[1]
        document = Document(page_content=content, metadata={'source': path,'org_article_name': result[1]})
        documents.append(document)
    return documents

    
if __name__ == '__main__':
    a = structual_text_v4(article_name="揽投部规范化管理口袋书—揽投部经理20250208")
    for i in a:
        print(i)